import requests, zipfile,StringIO
import urllib2


def DownloadAndUnzipCafef(path):
    #get the lastest zip data
    f = urllib2.urlopen("http://s.cafef.vn/du-lieu/download.chn")
    data = f.read()
    baseUrl = "http://images1.cafef.vn/data/"
    pos = data.find(baseUrl) + len(baseUrl)
    strDate = data[pos:pos+8]
    strDate2 = strDate[6:] + strDate[4:6] + strDate[:4]
        
    #url = ... http://images1.cafef.vn/data/20130705/CafeF.SolieuGD.Upto05072013.zip
    zip_file_urls =  [  baseUrl +"/"+strDate+"/CafeF.SolieuGD.Upto"+strDate2+".zip", 
                        baseUrl +"/"+ strDate+"/CafeF.Index.Upto"+strDate2+".zip"]
    
    #download zip data and import
    for url in zip_file_urls:    
        print "Downloading "+ url[url.rfind("/")+1:],
        try:        
            f = urllib2.urlopen(url)
            data = f.read()
            z = zipfile.ZipFile(StringIO.StringIO(data))
            z.extractall(path)
            print "DONE !"  
        except urllib2.HTTPError as e:
            print "\nCan not import : " + e.reason
            
    return 1
    
def ImportTradingDailyFrom(ffile):
    # if is index, need to add ^ to ticker
    sql = ""
    extraTicker = ""    
    stype =1;
    if(ffile.find("INDEX") > 0):
        extraTicker = "^"
        stype = 2
                
    ids = common.getSecurityIDs(db,stype)
    
    # 3. open file and start import    
    with open(ffile) as f:          # within "with" keyword, it handle the open/close file
        
        print "Start importing : %s" % ffile
        
        content = f.readlines()        
        numlines = len(content)
        
        for n in range(1,numlines):
            s = content[n].replace('\n','').split(",")
            
            ticker = extraTicker + s[0].upper()       
            if ticker not in ids:
                print "Error, your database do not have " + ticker
                continue        
             
            sec_id = ids[ticker][0]
            trading_destination_id = ids[ticker][1]            
            sdate = s[1][0:4] + "-" + s[1][4:6] + "-" + s[1][6:]
            open_prc = s[2]
            high_prc = s[3]
            low_prc = s[4]
            close_prc = s[5]
            volume = s[6]
            
            sql = "INSERT INTO `trading_daily_%s` VALUES ('%d','%d'" + ",'%s'" * 7 + ")"
            sql = sql % (s[1][0:4], sec_id ,trading_destination_id,sdate,volume,'0',open_prc,
                         high_prc,low_prc,close_prc)
            
            if(common.fetchSQL(db, sql) is not None):
                print ".",
        
        print "DONE !"
